package com.g2room.myuall.grap.arti;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.grap.univ.po.Article;
import com.g2room.myuall.util.DbUtil;
import com.g2room.myuall.util.HtmlUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrapLiuxue86Arti implements Runnable {

	private static final String source="留学出国网"; 
	private static List<String> detailUrls = new ArrayList<String>();
	private static Map<String, String> urlsMap = new HashMap<String, String>();
	private List<Article> artis;
	private static List<Article> artiss = new ArrayList<Article>();
	static {
		getSourceUrl();
		urlsMap.put("http://www.liuxue86.com/zhufuyu/map/", "祝福语");
//		urlsMap.put("http://www.liuxue86.com/gongzuozongjie/map/", "工作总结");
//		urlsMap.put("http://www.liuxue86.com/fanwen/map/", "范文");
//		urlsMap.put("http://www.liuxue86.com/hetongfanben/map/", "合同范本");
//		urlsMap.put("http://www.liuxue86.com/yanjianggao/map/", "演讲稿");
//		urlsMap.put("http://www.liuxue86.com/rudangshenqingshu/map/", "入党申请书");
//		urlsMap.put("http://www.liuxue86.com/rutuanshenqingshu/map/", "入团申请书");
//		urlsMap.put("http://www.liuxue86.com/dhg/map/", "读后感");
//		urlsMap.put("http://www.liuxue86.com/xindetihui/map/", "心得体会");
//		urlsMap.put("http://www.liuxue86.com/jiantaoshu/map/", "检讨书");
//		urlsMap.put("http://www.liuxue86.com/cehuashu/map/", "策划书");
//		urlsMap.put("http://www.liuxue86.com/jiejiari/map/", "节假日");
//		urlsMap.put("http://www.liuxue86.com/kouhao/map/", "口号栏目");
//		urlsMap.put("http://www.liuxue86.com/zhuchici/map/", "主持词");
//		urlsMap.put("http://www.liuxue86.com/yaoqinghan/map/", "邀请函");
//		urlsMap.put("http://www.liuxue86.com/liyi/map/", "礼仪");
//		urlsMap.put("http://www.liuxue86.com/gongzuojihua/map/", "工作计划书");
//		urlsMap.put("http://www.liuxue86.com/shixibaogao/map/", "实习报告");
//		urlsMap.put("http://www.liuxue86.com/huodongzongjie/map/", "活动总结");
//		urlsMap.put("http://www.liuxue86.com/shuzhibaogao/map/", "述职报告");
//		urlsMap.put("http://www.liuxue86.com/ziwojianding/map/", "自我鉴定");
//		urlsMap.put("http://www.liuxue86.com/ziwopingjia/map/", "自我评价");
//		urlsMap.put("http://www.liuxue86.com/jianli/map/", "简历");
//		urlsMap.put("http://www.liuxue86.com/cizhibaogao/map/", "辞职报告");
//		urlsMap.put("http://www.liuxue86.com/shouchaobao/map/", "手抄报");
//		urlsMap.put("http://www.liuxue86.com/jieshaoxin/map/", "介绍信");
		
//		urlsMap.put("http://zw.liuxue86.com/map/", "作文");
//		urlsMap.put("http://www.liuxue86.com/gaokao/map/", "高考");
//		urlsMap.put("http://www.liuxue86.com/zhongkao/map/", "中考");
//		urlsMap.put("http://www.liuxue86.com/gongwuyuan/map/", "公务员");
//		urlsMap.put("http://www.liuxue86.com/kaoyan/map/", "考研");
//		urlsMap.put("http://www.liuxue86.com/kuaiji/map/", "会计");
//		urlsMap.put("http://yimin.liuxue86.com/yimin/map/", "移民");
//		urlsMap.put("http://ielts.liuxue86.com/map/", "雅思");
//		urlsMap.put("http://toefl.liuxue86.com/map/", "托福");
//		urlsMap.put("http://gre.liuxue86.com/map/", "GRE");
//		urlsMap.put("http://english.liuxue86.com/map/", "英语");
		
	}
	public static void main(String[] args) throws Exception{
		GrapLiuxue86Arti gl1 = new GrapLiuxue86Arti();
		for(String url : urlsMap.keySet()) {
			gl1.getMap(url, urlsMap.get(url));
		}

 		System.out.println("===小类数量：" + artiss.size());
        for(int i = 0; i < artiss.size(); ) {
        	System.out.println("=============="+i);
        	GrapLiuxue86Arti gl = new GrapLiuxue86Arti();
//        	gl.getPage(artis.get(i));
        	gl.artis = artiss.subList(i, Math.min(i+100, artiss.size()));
        	i+=100;
        	Thread t = new Thread(gl);
        	t.start();
        }
	}
	
	/**
	 * 获取Map列表
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getMap(String url, String type) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setDescription("获取List错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		if(doc == null) return;
		
		Elements els = doc.select(".mar_10");
		if(null == els || els.isEmpty()) return;
 		for(Element el : els) {
			// 获取大分类
			Elements tel = el.select("h3 b a");
			if(null == tel || tel.isEmpty()) continue;
			String catName = tel.text();
			if(catName.contains("全部地图")) continue;
			
			Elements aels = el.select(".map_lj li a");
			if(null == els || els.isEmpty()) return;
			for(Element ael : aels) {
	 			Article arti = new Article();
	 			arti.setCategoryName(type);
	 			arti.setTypeName(catName);
				arti.setClassName(ael.text());
				arti.setSourceUrl(ael.attr("href"));
				
				artiss.add(arti);
			}
		}
	}
	
	/**
	 * 获取分页列表
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(Article arti) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(arti.getSourceUrl());
		} catch (IOException e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(arti.getSourceUrl());
			ge.setType("2");
			ge.setContent(JSON.toJSONString(arti));
			ge.setDescription("获取Page错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		if(doc == null) return;
		List<String> listUrls = new ArrayList<String>();
		Elements els = doc.select("#pages li a");
		listUrls.add(arti.getSourceUrl());
		if(null != els && !els.isEmpty()) {
			String text = els.get(els.size() - 2).text();
			try {
				Integer i = Integer.parseInt(text);
				for(int j = 2; j<i; j++) {
					listUrls.add(arti.getSourceUrl() + j + ".html");
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(String listUrl : listUrls) {
        	getList(listUrl, arti);
        }
	}
	
	/**
	 * 获取列表列表
	 */
	public void getList(String url, Article art) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType("3");
			ge.setContent(JSON.toJSONString(art));
			ge.setDescription("获取List错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		if(doc == null) return;
		Elements els = doc.select(".article_list ul li a");
		Elements dateels = doc.select(".article_list ul li .date");
		List<Article> artis = new ArrayList<Article>();
		if(null != els && !els.isEmpty()) {
			for(int i = 0; i<els.size(); i++) {
	 			Article arti = new Article();
	 			arti.setCategoryName(art.getCategoryName());
	 			arti.setTypeName(art.getTypeName());
				arti.setClassName(art.getClassName());
				arti.setSource(source);
				arti.setSourceUrl(els.get(i).attr("href"));
				arti.setTitle(els.get(i).text());
				arti.setPublishTime(dateels.get(i).text());
				if(!detailUrls.contains(arti.getSourceUrl())) {
					detailUrls.add(arti.getSourceUrl());
					artis.add(arti);
				}
			}
		}

		List<Article> artiss = new ArrayList<Article>();
        for(Article arti : artis) {
        	Article ar = getDetail(arti);
        	if(null != ar) artiss.add(ar);
        }
        if(artiss.size() > 0) insertArtiDb(artiss);
	}
	
	/**
	 * 获取列表列表
	 */
	public Article getDetail(Article arti) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(arti.getSourceUrl());
		} catch (IOException e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setUrl(arti.getSourceUrl());
			ge.setType("4");
			ge.setContent(JSON.toJSONString(arti));
			ge.setDescription("获取Detail错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		if(doc == null) return null;
		Elements els = doc.select(".main_zhengw");
		if(null != els && !els.isEmpty()) {
			Elements tels = els.first().children();
			StringBuffer content = new StringBuffer();
			int start = 0;
			for(Element el : tels) {
				if(el.tagName().equalsIgnoreCase("p")) {
					start++;
				}
				if(start < 4 && (el.tagName().equalsIgnoreCase("script") || el.text().contains("小编精心推荐阅读") || el.text().contains("推荐阅读"))) {
					start = -1;
				}
				if(start > 2 && (el.tagName().equalsIgnoreCase("script") || el.text().contains("小编精心推荐阅读") || el.text().contains("推荐阅读"))) {
					start = 0;
				}
				
				if(start > 0) {
					content.append(el.outerHtml());
				}
			}
			
			arti.setContent(HtmlUtil.delAllTag(content.toString())
					.replaceAll("出国留学网", "高考信息网")
					.replaceAll("http://www.liuxue86.com", "myuall.com")
					.replaceAll("/uploadfile/", "http://www.liuxue86.com/uploadfile/"));
		}
		return arti;
	}
	

	public void insertArtiDb(List<Article> artis) {
		String sql = "insert into data_doc(category_name,type_name,content,source_url,class_name,source,title,publish_time) values(?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getConn().prepareStatement(sql);
			// insert into arti(category_name,type_name,content,source_url,class_name,source,title,hot,state,publish_time) values(?,?,?,?,?,?,?,?,?,?) 18696831689
			for(Article cc : artis) {
				ps.setString(1, cc.getCategoryName());
				ps.setString(2, cc.getTypeName());
				ps.setString(3, cc.getContent());
				ps.setString(4, cc.getSourceUrl());
				ps.setString(5, cc.getClassName());
				ps.setString(6, cc.getSource());
				ps.setString(7, cc.getTitle());
				ps.setString(8, cc.getPublishTime());
				ps.addBatch();
			}
			ps.executeBatch();	
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setType("5");
			ge.setContent(JSON.toJSONString(artis));
			ge.setDescription("获取DB错误");
			insert2ErrDb(ge);
		
		} finally {
			if(null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getConn().prepareStatement("insert into grap_err(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public static void getSourceUrl() {
		try {
			Connection connection = DbUtil.getConn();
			Statement statement = connection.createStatement();
			String sql = "select source_url from data_doc";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				detailUrls.add(rs.getString("source_url"));
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}

	@Override
	public void run() {
        for(Article arti : artis) {
        	getPage(arti);
        }
	}
}
